register dist/baconsnake.jar;
register lib/jython.jar;

raw = LOAD 'data/excite-small.log' USING PigStorage('\t') AS (user:chararray, time:chararray, query:chararray);

houred = FOREACH raw GENERATE user, baconsnake.EvalFunction('ExtractHour', 'def ExtractHour(timestamp):\n\treturn timestamp[6:8]\n', time) as hour, query;

hour_group = GROUP houred BY hour;

hour_frequency = FOREACH hour_group GENERATE group as hour, baconsnake.EvalFunction('AvgLength', 'def AvgLength(grp):\n\tsum = 0\n\tfor item in grp:\n\t\tif len(item) > 0:\n\t\t\tsum = sum + len(item[0])\t\n\treturn str(sum / len(grp))\n', $1.query) as count;

DUMP hour_frequency;

-- The excite query log timestamp format is YYMMDDHHMMSS
-- This function extracts the hour, HH
-- Returns average length of bag

